import requests
import redis
from lxml import etree
import time
import json
import pymysql
from DBUtils.PersistentDB import PersistentDB
sess=requests.session()
pool1 = PersistentDB(creator=pymysql, maxusage=None, ping=0, closeable=False, host='127.0.0.1', user='root',
                         password='root', db='blog_spider', charset='utf8')
headers={
    'cookie':'uuid_tt_dd=10_35506074690-1543408047225-128682; _ga=GA1.2.1747139185.1546238144; __yadk_uid=YTIMDOf9Dj1604cMXk1BIQRtSaEtuoC7; bdshare_firstime=1552953352430; UN=fanfzj; BT=1553358121270; dc_session_id=10_1553869355729.263729; smidV2=201812021611467b57a701e3d1ab90103734fe8cb98c5300121369440d44160; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1554650343,1554650343,1554650343,1557066477; Hm_ct_6bcd52f51e9b3dce32bec4a3997715ac=6525*1*10_35506074690-1543408047225-128682!5744*1*fanfzj; c-login-auto=15; TY_SESSION_ID=56cbc0bc-633b-444f-90ef-36d55e0e4c42; SESSION=97fc1b4d-5a37-4230-a72e-2b5bccbcf684; dc_tos=preblk; ADHOC_MEMBERSHIP_CLIENT_ID1.0=80c382a2-1139-938a-2cb5-1224f4edeaea',
    'referer':'https://blog.csdn.net/',
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.63 Safari/537.36 Qiyu/2.1.1.1'
}
redis_pool=redis.ConnectionPool(host='47.104.82.16',db=4,password='FanTan879425')
r=redis.Redis(connection_pool=redis_pool)
while True:
    # 连接数据库
    conn = pool1.connection()
    cursor = conn.cursor()
    res=r.spop('csdn_url')
    if not res or res is None:
        continue
    res=json.loads(res.decode('utf-8'))
    print(res)
    url=res['url']
    result=sess.get(url,headers=headers).text
    sel=etree.HTML(result)
    # fbrq
    date=sel.xpath('//span[@class="time"]/text()')[0]
    print(date)
    res['fbrq']=time.strftime('%Y-%m-%d',time.strptime(date,'%Y年%m月%d日 %H:%M:%S'))
    # read-count
    res['read_count']=sel.xpath('//span[@class="read-count"]/text()')[0].strip('阅读数：')
    # content
    res['contents']=sel.xpath('string(//*[@id="content_views"])')
    sql = 'insert into csdn values(NULL,%s,%s,%s,%s,%s,%s,%s,%s)'
    cursor.execute(sql, (
    res['title'], res['contents'], res['author'], res['category'], res['tag'], res['fbrq'],res['read_count'], url))
    conn.commit()


